{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    " \n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from sklearn.externals import joblib \n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.linear_model import RidgeCV\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "import xgboost as xgb\n",
    "from lightgbm import LGBMRegressor\n",
    "import math\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据预处理\n",
    "def clean_data(train_data_X):    \n",
    "    for j in range(len(np.array(train_data_X.columns))):\n",
    "        if train_data_X.columns[j] != \"ID\":\n",
    "            print(\"正在处理第%d列数据\"%j)\n",
    "            array = train_data_X[train_data_X.columns[j]]\n",
    "            num = 0\n",
    "            len_ = len(train_data_X[train_data_X.columns[j]])\n",
    "            for i in range(len_):   \n",
    "                std_ = array.std()\n",
    "                min_ = array.min()\n",
    "                max_ = array.max()\n",
    "                mean_ = array.mean()\n",
    "                num = array[i]\n",
    "                std_max =np.float(mean_+3*std_)\n",
    "                std_min = np.float(mean_-3*std_)\n",
    "                if (array[0]>std_max) | (array[0]<std_min):\n",
    "    #                 mean_del = mean_-(array[0]-mean_)/len_\n",
    "    #                 array[0] = mean_del\n",
    "                    array[0] = mean_\n",
    "                if (array[len_-1]>std_max) | (array[len_-1]<std_min):\n",
    "                    array[len_-1] = mean_\n",
    "                if i>0 and i<(len_-1):\n",
    "                    if (array[i]>std_max) | (array[i]<std_min):\n",
    "                        if (array[i-1]<std_max) and (array[i-1]>std_min) and (array[i+1]<std_max) and (array[i+1]>std_min):\n",
    "                            array[i]= np.float(array[i-1]+array[i+1])/2\n",
    "                        elif ((array[i-1]<std_max) and (array[i-1]>std_min))| (array[i+1]>std_max) | (array[i+1]<std_min): \n",
    "                            array[i] = array[i-1]\n",
    "                        elif (array[i-1]<std_max) | (array[i-1]>std_min)|((array[i+1]<std_max) and (array[i+1]>std_min)):\n",
    "                            array[i] = array[i+1]\n",
    "            train_data_X[train_data_X.columns[j]] = array\n",
    "        else:\n",
    "            pass\n",
    "            \n",
    "    return train_data_X\n",
    "def drop_all_outlier(df):\n",
    "    df.drop_duplicates(df.columns.drop('ID'), keep='first', inplace=True)\n",
    "    df.drop(df[(df.电压A > 800) | (df.电压A < 500)].index,inplace=True)\n",
    "    df.drop(df[(df.电压B > 800) | (df.电压B < 500)].index,inplace=True)\n",
    "    df.drop(df[(df.电压C > 800) | (df.电压C < 500)].index,inplace=True)\n",
    "    df.drop(df[(df.现场温度 > 30) | (df.现场温度 < -30)].index,inplace=True)\n",
    "    df.drop(df[(df.转换效率A > 100)].index,inplace=True)\n",
    "    df.drop(df[(df.转换效率B > 100)].index,inplace=True)\n",
    "    df.drop(df[(df.转换效率C > 100)].index,inplace=True)\n",
    "    df.drop(df[(df.风向 > 360)].index,inplace=True)\n",
    "    df.drop(df[(df.风速 > 20)].index,inplace=True)\n",
    "    return df\n",
    "# 生成数据\n",
    "def generate_train_data(train_data, test_data, poly=False, select=False):\n",
    "    y = train_data['发电量']\n",
    "    X = train_data.drop(['发电量','ID'], axis=1)\n",
    "    sub_data = test_data.drop(['ID'], axis=1)\n",
    "    \n",
    "    polynm = None\n",
    "    if poly:\n",
    "        from sklearn.preprocessing import PolynomialFeatures\n",
    "        polynm = PolynomialFeatures(degree=2, interaction_only=True)\n",
    "        X = polynm.fit_transform(X)\n",
    "        sub_data = polynm.transform(sub_data)\n",
    "        \n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)\n",
    "    \n",
    "    sm = None\n",
    "    if select:\n",
    "        from sklearn.feature_selection import SelectFromModel\n",
    "        sm = SelectFromModel(GradientBoostingRegressor(random_state=2))\n",
    "        X_train = sm.fit_transform(X_train, y_train)\n",
    "        X_test = sm.transform(X_test)\n",
    "        sub_data = sm.transform(sub_data)\n",
    "        \n",
    "    return X_train, X_test, y_train, y_test, sub_data, sm, polynm\n",
    "\n",
    "def cal_score(mse):\n",
    "    if isinstance(mse, float):\n",
    "        return 1 / (1 + math.sqrt(mse))\n",
    "    else:\n",
    "        return np.divide(1, 1 + np.sqrt(mse))\n",
    "#  定义交叉验证函数  \n",
    "def cross_validation_test(models, train_X_data, train_y_data, cv=5):\n",
    "    model_name, mse_avg, score_avg = [], [], []\n",
    "    for i, model in enumerate(models):\n",
    "        print(i + 1,'- Model:', str(model).split('(')[0])\n",
    "        model_name.append(str(i + 1) + '.' + str(model).split('(')[0])\n",
    "        nmse = cross_val_score(model, train_X_data[i], train_y_data[i], cv=cv, scoring='neg_mean_squared_error')\n",
    "        avg_mse = np.average(-nmse)\n",
    "        scores = cal_score(-nmse)\n",
    "        avg_score = np.average(scores)\n",
    "        mse_avg.append(avg_mse)\n",
    "        score_avg.append(avg_score)\n",
    "        print('MSE:', -nmse)\n",
    "        print('Score:', scores)\n",
    "        print('Average XGB - MSE:', avg_mse, ' - Score:', avg_score, '\\n')\n",
    "    res = pd.DataFrame()\n",
    "    res['Model'] = model_name\n",
    "    res['Avg MSE'] = mse_avg\n",
    "    res['Avg Score'] = score_avg\n",
    "    return res\n",
    "\n",
    "\n",
    "def add_avg(df):\n",
    "    array = np.array(df[\"平均功率\"])\n",
    "    newarray=[]\n",
    "    num = 0\n",
    "    for i in np.arange(len(array)):\n",
    "        for j in np.arange(10):\n",
    "            sum_1=0\n",
    "            for m in np.arange(j):\n",
    "                sum_1 += array[m]\n",
    "            if i<10:\n",
    "                num = sum_1/(j+1)\n",
    "            else:\n",
    "                num = (array[i-1]+array[i-2]+array[i-3]+array[i-4]+array[i-5]+array[i-6]+array[i-7]+array[i-8]+array[i-9])/9\n",
    "        newarray.append(num)\n",
    "    df[\"old平均功率\"] = newarray\n",
    "    return df\n",
    "\n",
    "# 对上面的函数 add_avg(df) 做了一层封装\n",
    "# 输入 矩阵df 源列名str1 目标列名str2\n",
    "# 输出 增加一列历史数据均值的矩阵\n",
    "def add_avgs(df,str1,str2):\n",
    "    array = np.array(df[str1])\n",
    "    newarray=[]\n",
    "    num = 0\n",
    "    for i in np.arange(len(array)):\n",
    "        for j in np.arange(10):\n",
    "            sum_1=0\n",
    "            for m in np.arange(j):\n",
    "                sum_1 += array[m]\n",
    "            if i<10:\n",
    "                num = sum_1/(j+1)\n",
    "            else:\n",
    "                num = (array[i-1]+array[i-2]+array[i-3]+array[i-4]+array[i-5]+array[i-6]+array[i-7]+array[i-8]+array[i-9])/9\n",
    "        newarray.append(num)\n",
    "    df[str2] = newarray\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.read_csv('./data/public.train.csv')\n",
    "test_data = pd.read_csv('./data/public.test.csv')\n",
    "\n",
    "df_result = pd.DataFrame()\n",
    "df_result['ID'] = list(test_data['ID'])\n",
    "special_missing_ID = test_data[test_data[(test_data == 0) | (test_data == 0.)].count(axis=1) > 13]['ID']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 异常值处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:8: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=True'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass sort=False\n",
      "\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "cleaned_train_data = train_data.copy()\n",
    "cleaned_train_data = drop_all_outlier(cleaned_train_data)\n",
    "\n",
    "cleaned_sub_data = test_data.copy()\n",
    "cleaned_sub_data = drop_all_outlier(cleaned_sub_data)\n",
    "cleaned_sub_data_ID = cleaned_sub_data['ID']\n",
    "\n",
    "all_data  = pd.concat([train_data, test_data], axis=0).sort_values(by='ID').reset_index().drop(['index'], axis=1)\n",
    "bad_feature = ['ID','功率A', '功率B', '功率C', '平均功率', '现场温度', '电压A', '电压B', '电压C', '电流B', '电流C', '转换效率', '转换效率A', '转换效率B', '转换效率C']\n",
    "bad_index1 = all_data[bad_feature][\n",
    "    (all_data[bad_feature] > all_data[bad_feature].mean() + 2 * all_data[bad_feature].std()) | \n",
    "    (all_data[bad_feature] < all_data[bad_feature].mean() - 2 * all_data[bad_feature].std())\n",
    "].dropna(how='all').index\n",
    "bad_index2 = all_data[\n",
    "    ((all_data['电压A']<500)&(all_data['电压A']!=0))|\n",
    "    ((all_data['电压B']<500)&(all_data['电压B']!=0))|\n",
    "    ((all_data['电压C']<500)&(all_data['电压C']!=0))].index\n",
    "bad_index = pd.Int64Index(list(bad_index1)+list(bad_index2))\n",
    "# all_data.loc[np.concatenate([bad_index -1,bad_index,bad_index+1])].sort_values(by='ID', ascending=True)\n",
    "\n",
    "\n",
    "nn_bad_data = all_data.loc[np.concatenate([bad_index - 1, bad_index, bad_index + 1])].sort_values(by='ID', ascending=True).drop_duplicates()\n",
    "bad_data = all_data.loc[bad_index].sort_values(by='ID', ascending=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 上下记录均值替代异常值\n",
    "for idx, line in bad_data.iterrows():\n",
    "    ID = line['ID']\n",
    "    col_index = line[bad_feature][ \n",
    "        (line[bad_feature] > all_data[bad_feature].mean() + 3 * all_data[bad_feature].std())| \n",
    "        (line[bad_feature] < all_data[bad_feature].mean() - 3 * all_data[bad_feature].std())\n",
    "    ].index\n",
    "    index = all_data[all_data['ID'] == ID].index\n",
    "    \n",
    "    before_offset = 1\n",
    "    while (idx + before_offset)in bad_index:\n",
    "        before_offset += 1\n",
    "\n",
    "    after_offset = 1\n",
    "    while (idx + after_offset) in bad_index:\n",
    "        after_offset += 1\n",
    "    \n",
    "    replace_value = (all_data.loc[index - before_offset, col_index].values + all_data.loc[index + after_offset, col_index].values) / 2\n",
    "    all_data.loc[index, col_index] = replace_value[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 拆分数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#拆分数据\n",
    "train_data = all_data.drop(all_data[all_data['ID'].isin(df_result['ID'])].index).reset_index().drop(['index'], axis=1)\n",
    "test_data = all_data[all_data['ID'].isin(df_result['ID'])].drop(['发电量'], axis=1).reset_index().drop(['index'], axis=1)\n",
    "len(train_data), len(test_data)\n",
    "# 去除重复值\n",
    "train_data = train_data.drop_duplicates(train_data.columns.drop('ID'), keep='first')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 增加 历史平均功率\n",
    "train_data = add_avg(train_data)\n",
    "test_data = add_avg(test_data)\n",
    "cleaned_train_data = add_avg(cleaned_train_data)\n",
    "cleaned_sub_data = add_avg(cleaned_sub_data)\n",
    "\n",
    "# 增加 历史平均功率A 历史平均功率B 历史平均功率C \n",
    "train_data = add_avgs(train_data,\"功率A\",\"功率Ahistory\")\n",
    "train_data = add_avgs(train_data,\"功率B\",\"功率Bhistory\")\n",
    "train_data = add_avgs(train_data,\"功率C\",\"功率Chistory\")\n",
    "\n",
    "test_data = add_avgs(test_data,\"功率A\",\"功率Ahistory\")\n",
    "test_data = add_avgs(test_data,\"功率B\",\"功率Bhistory\")\n",
    "test_data = add_avgs(test_data,\"功率C\",\"功率Chistory\")\n",
    "\n",
    "cleaned_train_data = add_avgs(cleaned_train_data,\"功率A\",\"功率Ahistory\")\n",
    "cleaned_train_data = add_avgs(cleaned_train_data,\"功率B\",\"功率Bhistory\")\n",
    "cleaned_train_data = add_avgs(cleaned_train_data,\"功率C\",\"功率Chistory\")\n",
    "\n",
    "cleaned_sub_data = add_avgs(cleaned_sub_data,\"功率A\",\"功率Ahistory\")\n",
    "cleaned_sub_data = add_avgs(cleaned_sub_data,\"功率B\",\"功率Bhistory\")\n",
    "cleaned_sub_data = add_avgs(cleaned_sub_data,\"功率C\",\"功率Chistory\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test, sub_data, sm, polynm = generate_train_data(train_data, test_data, poly=True, select=True)\n",
    "\n",
    "clean_X_train, clean_X_test, clean_y_train, clean_y_test, clean_sub_data, _, _ = generate_train_data(cleaned_train_data, cleaned_sub_data, poly=False, select=False)\n",
    "\n",
    "clean_X = np.concatenate([clean_X_train, clean_X_test])\n",
    "clean_y = np.concatenate([clean_y_train, clean_y_test])\n",
    "clean_X = polynm.transform(clean_X)\n",
    "clean_X = sm.transform(clean_X)\n",
    "\n",
    "clean_sub_data = polynm.transform(clean_sub_data)\n",
    "clean_sub_data = sm.transform(clean_sub_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stacking Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tree Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_X_train = np.concatenate([X_train, X_test])\n",
    "all_y_train = np.concatenate([y_train, y_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 - Model: XGBRegressor\n",
      "MSE: [0.00930431 0.02146953 0.01092385 0.08733146 0.01410529]\n",
      "Score: [0.91202693 0.87220091 0.90537294 0.77189144 0.8938422 ]\n",
      "Average XGB - MSE: 0.028626886898687677  - Score: 0.871066884636692 \n",
      "\n",
      "2 - Model: XGBRegressor\n",
      "MSE: [0.00930045 0.02143601 0.01088144 0.08727448 0.01409997]\n",
      "Score: [0.91204358 0.87228795 0.90553944 0.77194889 0.89386009]\n",
      "Average XGB - MSE: 0.028598471225021054  - Score: 0.871135990482187 \n",
      "\n",
      "3 - Model: XGBRegressor\n",
      "MSE: [0.00928128 0.02140966 0.01087166 0.0872756  0.01411371]\n",
      "Score: [0.91212631 0.87235645 0.9055779  0.77194776 0.89381391]\n",
      "Average XGB - MSE: 0.028590380387984265  - Score: 0.8711644658848126 \n",
      "\n",
      "4 - Model: GradientBoostingRegressor\n",
      "MSE: [0.01021152 0.01586433 0.01014427 0.08261867 0.01575506]\n",
      "Score: [0.90822228 0.88813601 0.90849727 0.77673846 0.88847889]\n",
      "Average XGB - MSE: 0.026918768710868807  - Score: 0.8740145820017053 \n",
      "\n",
      "5 - Model: GradientBoostingRegressor\n",
      "MSE: [0.01019475 0.01585153 0.01013239 0.08259549 0.01574189]\n",
      "Score: [0.90829073 0.88817608 0.90854596 0.77676279 0.88852031]\n",
      "Average XGB - MSE: 0.026903210969190343  - Score: 0.8740591749181554 \n",
      "\n",
      "6 - Model: GradientBoostingRegressor\n",
      "MSE: [0.01019465 0.01585533 0.01013889 0.08260659 0.01574588]\n",
      "Score: [0.90829115 0.88816419 0.90851933 0.77675114 0.88850775]\n",
      "Average XGB - MSE: 0.026908267815356567  - Score: 0.8740467110606437 \n",
      "\n",
      "7 - Model: RandomForestRegressor\n",
      "MSE: [0.01173158 0.02217058 0.01312735 0.08732022 0.01579952]\n",
      "Score: [0.90227267 0.87039936 0.89720324 0.77190277 0.88833919]\n",
      "Average XGB - MSE: 0.030029852022702606  - Score: 0.8660234455334705 \n",
      "\n",
      "8 - Model: RandomForestRegressor\n",
      "MSE: [0.01185825 0.02238467 0.01326815 0.08823406 0.01594928]\n",
      "Score: [0.90179817 0.86985638 0.89671022 0.77098495 0.88787045]\n",
      "Average XGB - MSE: 0.030338880216229956  - Score: 0.8654440333552019 \n",
      "\n",
      "9 - Model: RandomForestRegressor\n",
      "MSE: [0.01189993 0.02216936 0.01325616 0.08785864 0.0160992 ]\n",
      "Score: [0.90164268 0.87040249 0.89675208 0.77136116 0.88740387]\n",
      "Average XGB - MSE: 0.030256657481180817  - Score: 0.8655124562208636 \n",
      "\n",
      "10 - Model: LGBMRegressor\n",
      "MSE: [0.01167854 0.02169127 0.01104104 0.08598136 0.01450679]\n",
      "Score: [0.90247227 0.87162714 0.90491483 0.77326018 0.89250323]\n",
      "Average XGB - MSE: 0.028979800074772542  - Score: 0.8689555293391557 \n",
      "\n",
      "11 - Model: LGBMRegressor\n",
      "MSE: [0.01181071 0.02274185 0.01113773 0.08592376 0.0152823 ]\n",
      "Score: [0.9019759  0.8689577  0.90453907 0.77331892 0.88997933]\n",
      "Average XGB - MSE: 0.02937926956077757  - Score: 0.8677541830461308 \n",
      "\n",
      "12 - Model: LGBMRegressor\n",
      "MSE: [0.01252845 0.02234874 0.01083608 0.08476703 0.0155606 ]\n",
      "Score: [0.89933673 0.86994727 0.90571794 0.77450468 0.88909268]\n",
      "Average XGB - MSE: 0.029208181041961313  - Score: 0.8677198591114668 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Avg MSE</th>\n",
       "      <th>Avg Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.XGBRegressor</td>\n",
       "      <td>0.028627</td>\n",
       "      <td>0.871067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.XGBRegressor</td>\n",
       "      <td>0.028598</td>\n",
       "      <td>0.871136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.XGBRegressor</td>\n",
       "      <td>0.028590</td>\n",
       "      <td>0.871164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.GradientBoostingRegressor</td>\n",
       "      <td>0.026919</td>\n",
       "      <td>0.874015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.GradientBoostingRegressor</td>\n",
       "      <td>0.026903</td>\n",
       "      <td>0.874059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6.GradientBoostingRegressor</td>\n",
       "      <td>0.026908</td>\n",
       "      <td>0.874047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7.RandomForestRegressor</td>\n",
       "      <td>0.030030</td>\n",
       "      <td>0.866023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8.RandomForestRegressor</td>\n",
       "      <td>0.030339</td>\n",
       "      <td>0.865444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9.RandomForestRegressor</td>\n",
       "      <td>0.030257</td>\n",
       "      <td>0.865512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10.LGBMRegressor</td>\n",
       "      <td>0.028980</td>\n",
       "      <td>0.868956</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11.LGBMRegressor</td>\n",
       "      <td>0.029379</td>\n",
       "      <td>0.867754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12.LGBMRegressor</td>\n",
       "      <td>0.029208</td>\n",
       "      <td>0.867720</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          Model   Avg MSE  Avg Score\n",
       "0                1.XGBRegressor  0.028627   0.871067\n",
       "1                2.XGBRegressor  0.028598   0.871136\n",
       "2                3.XGBRegressor  0.028590   0.871164\n",
       "3   4.GradientBoostingRegressor  0.026919   0.874015\n",
       "4   5.GradientBoostingRegressor  0.026903   0.874059\n",
       "5   6.GradientBoostingRegressor  0.026908   0.874047\n",
       "6       7.RandomForestRegressor  0.030030   0.866023\n",
       "7       8.RandomForestRegressor  0.030339   0.865444\n",
       "8       9.RandomForestRegressor  0.030257   0.865512\n",
       "9              10.LGBMRegressor  0.028980   0.868956\n",
       "10             11.LGBMRegressor  0.029379   0.867754\n",
       "11             12.LGBMRegressor  0.029208   0.867720"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgbt1 = xgb.XGBRegressor(silent=1,max_leaf_nodes=255,seed=789,eta=0.03,scoring='neg_mean_squared_error',n_estimators=351, max_depth=5, max_features='sqrt', random_state=777, n_jobs=12)\n",
    "xgbt2 = xgb.XGBRegressor(silent=1,max_leaf_nodes=255,seed=1000,eta=0.03,scoring='neg_mean_squared_error',n_estimators=361, max_depth=5, max_features='sqrt', random_state=999, n_jobs=12)\n",
    "xgbt3 = xgb.XGBRegressor(silent=1,max_leaf_nodes=255,seed=1515,eta=0.03,scoring='neg_mean_squared_error',n_estimators=371, max_depth=5, max_features='sqrt', random_state=367, n_jobs=12)\n",
    "\n",
    "# n_estimators=1000  max_depth=5  'sqrt'  GradientBoostingRegressor 最佳参数 ,learning_rate=0.08\n",
    "gbdt1 = GradientBoostingRegressor(n_estimators=1060, max_depth=5, max_features='log2', random_state=789,learning_rate=0.08)\n",
    "gbdt2 = GradientBoostingRegressor(n_estimators=1100, max_depth=5, max_features='log2', random_state=123,learning_rate=0.08)\n",
    "gbdt3 = GradientBoostingRegressor(n_estimators=1090, max_depth=5, max_features='log2', random_state=1999,learning_rate=0.08)\n",
    "\n",
    "# n_estimators=700, max_features='auto', random_state=2, n_jobs=8,max_depth=10\n",
    "forest1 = RandomForestRegressor(n_estimators=740, max_features='sqrt', random_state=7, n_jobs=12)\n",
    "forest2 = RandomForestRegressor(n_estimators=730, max_features='sqrt', random_state=9, n_jobs=12)\n",
    "forest3 = RandomForestRegressor(n_estimators=745, max_features='sqrt', random_state=11, n_jobs=12) \n",
    "\n",
    "# n_estimators 850 num_leaves=14,learning_rate=0.1 max_depth=5  seed\n",
    "lgb1 = LGBMRegressor(n_estimators=840, max_depth=4, random_state=789, n_jobs=12,num_leaves=14,learning_rate=0.08,seed=666) \n",
    "lgb2 = LGBMRegressor(n_estimators=845, max_depth=4, random_state=798, n_jobs=12,num_leaves=13,learning_rate=0.1,seed=777)\n",
    "lgb3 = LGBMRegressor(n_estimators=850, max_depth=4, random_state=777, n_jobs=12,num_leaves=10,learning_rate=0.1,seed=999)\n",
    "\n",
    "cross_validation_test(\n",
    "    models=[    \n",
    "        xgbt1, xgbt2, xgbt3,\n",
    "        gbdt1, gbdt2, gbdt3,\n",
    "        forest1, forest2, forest3,\n",
    "        lgb1, lgb2, lgb3\n",
    "    ],\n",
    "    train_X_data=[\n",
    "        all_X_train, all_X_train, all_X_train, all_X_train,\n",
    "        all_X_train, all_X_train, all_X_train, all_X_train,\n",
    "        all_X_train, all_X_train, all_X_train, all_X_train\n",
    "    ],\n",
    "    train_y_data=[\n",
    "        all_y_train, all_y_train, all_y_train, all_y_train,\n",
    "        all_y_train, all_y_train, all_y_train, all_y_train,\n",
    "        all_y_train, all_y_train, all_y_train, all_y_train\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "regrs = [\n",
    "    xgbt1, gbdt1, forest1, lgb1,\n",
    "     xgbt2, gbdt2, forest2, lgb2,\n",
    "     xgbt3, gbdt3, forest3, lgb3\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Stacker(object):\n",
    "    def __init__(self, n_splits, stacker, base_models):\n",
    "        self.n_splits = n_splits\n",
    "        self.stacker = stacker\n",
    "        self.base_models = base_models\n",
    "    \n",
    "    # X: 原始训练集, y: 原始训练集真实值, predict_data: 原始待预测数据\n",
    "    def fit_predict(self, X, y, predict_data):\n",
    "        X = np.array(X)\n",
    "        y = np.array(y)\n",
    "        T = np.array(predict_data)\n",
    "\n",
    "        folds = list(KFold(n_splits=self.n_splits, shuffle=False, random_state=2018).split(X, y))\n",
    "        \n",
    "        # 以基学习器预测结果为特征的 stacker的训练数据 与 stacker预测数据\n",
    "        S_train = np.zeros((X.shape[0], len(self.base_models)))\n",
    "        S_predict = np.zeros((T.shape[0], len(self.base_models)))\n",
    "        \n",
    "        for i, regr in enumerate(self.base_models):\n",
    "            print(i + 1, 'Base model:', str(regr).split('(')[0])\n",
    "            S_predict_i = np.zeros((T.shape[0], self.n_splits))\n",
    "            \n",
    "            for j, (train_idx, test_idx) in enumerate(folds):\n",
    "                # 将X分为训练集与测试集\n",
    "                X_train, y_train, X_test, y_test = X[train_idx], y[train_idx], X[test_idx], y[test_idx]\n",
    "                print ('Fit fold', (j+1), '...')\n",
    "                regr.fit(X_train, y_train)\n",
    "                y_pred = regr.predict(X_test)                \n",
    "                S_train[test_idx, i] = y_pred\n",
    "                S_predict_i[:, j] = regr.predict(T)\n",
    "            \n",
    "            S_predict[:, i] = S_predict_i.mean(axis=1)\n",
    "\n",
    "        nmse_score = cross_val_score(self.stacker, S_train, y, cv=5, scoring='neg_mean_squared_error')\n",
    "        print('CV MSE:', -nmse_score)\n",
    "        print('Stacker AVG MSE:', -nmse_score.mean(), 'Stacker AVG Score:', np.mean(np.divide(1, 1 + np.sqrt(-nmse_score))))\n",
    "\n",
    "        self.stacker.fit(S_train, y)\n",
    "        res = self.stacker.predict(S_predict)\n",
    "        return res, S_train, S_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 Base model: XGBRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "2 Base model: GradientBoostingRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "3 Base model: RandomForestRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "4 Base model: LGBMRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "5 Base model: XGBRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "6 Base model: GradientBoostingRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "7 Base model: RandomForestRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "8 Base model: LGBMRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "9 Base model: XGBRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "10 Base model: GradientBoostingRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "11 Base model: RandomForestRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "12 Base model: LGBMRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "CV MSE: [0.0082839  0.0166939  0.00812777 0.08443426 0.01335088]\n",
      "Stacker AVG MSE: 0.026178142957411193 Stacker AVG Score: 0.8781454455688016\n"
     ]
    }
   ],
   "source": [
    "# stacking_mode1 = Ridge(alpha=0.008, copy_X=True, fit_intercept=False, solver='auto', random_state=2)\n",
    "# 用SVR融合是一个关键的步骤\n",
    "stacking_model = SVR(C=100, gamma=0.01, epsilon=0.01)\n",
    "stacker = Stacker(5, stacking_model, regrs)\n",
    "pred_stack, S_train_data, S_predict_data = stacker.fit_predict(all_X_train, all_y_train, sub_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 Base model: XGBRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "2 Base model: GradientBoostingRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "3 Base model: RandomForestRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "4 Base model: LGBMRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "5 Base model: XGBRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "6 Base model: GradientBoostingRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "7 Base model: RandomForestRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "8 Base model: LGBMRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "9 Base model: XGBRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "10 Base model: GradientBoostingRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "11 Base model: RandomForestRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "12 Base model: LGBMRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "CV MSE: [0.00897474 0.01787121 0.05824673 0.01123832 0.01017456]\n",
      "Stacker AVG MSE: 0.0213011117417595 Stacker AVG Score: 0.8827291568496861\n"
     ]
    }
   ],
   "source": [
    "stacking_model2 = SVR(C=100, gamma=0.01, epsilon=0.01)\n",
    "stacker2 = Stacker(5, stacking_model2, regrs)\n",
    "pred_clean_stack, S_clean_train_data, S_clean_predict_data = stacker2.fit_predict(clean_X, clean_y, clean_sub_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_result['score'] = pred_stack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 对结果用固定值进行修改 b榜过拟合，可以删掉\n",
    "index = df_result[df_result['ID'].isin(special_missing_ID)].index\n",
    "df_result.loc[index, 'score'] = 0.379993053"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "c_index = df_result[df_result['ID'].isin(cleaned_sub_data_ID)].index\n",
    "df_result.loc[c_index, 'score'] = pred_clean_stack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result.to_csv('submit.csv', index=False, header=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
